{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "58fc4b41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LlamaConfig {\n",
       "  \"attention_bias\": false,\n",
       "  \"attention_dropout\": 0.0,\n",
       "  \"bos_token_id\": 1,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"hidden_act\": \"silu\",\n",
       "  \"hidden_size\": 256,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 768,\n",
       "  \"max_position_embeddings\": 2048,\n",
       "  \"mlp_bias\": false,\n",
       "  \"model_type\": \"llama\",\n",
       "  \"num_attention_heads\": 16,\n",
       "  \"num_hidden_layers\": 4,\n",
       "  \"num_key_value_heads\": 8,\n",
       "  \"pretraining_tp\": 1,\n",
       "  \"rms_norm_eps\": 1e-06,\n",
       "  \"rope_scaling\": null,\n",
       "  \"rope_theta\": 10000.0,\n",
       "  \"tie_word_embeddings\": false,\n",
       "  \"transformers_version\": \"4.41.2\",\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 32000\n",
       "}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://blog.csdn.net/python12345678_/article/details/139798755\n",
    "from transformers import AutoConfig\n",
    "\n",
    "\n",
    "config = AutoConfig.for_model(\n",
    "    model_type='llama',\n",
    "    hidden_size=256,\n",
    "    intermediate_size=768,\n",
    "    num_attention_heads=16,\n",
    "    num_hidden_layers=4,\n",
    "    num_key_value_heads=8\n",
    ")\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a885eca9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LlamaTokenizerFast(name_or_path='./llama', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
       "\t0: AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t1: AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t2: AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 分词器\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('./llama')\n",
    "\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4e26fe76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LlamaForCausalLM(\n",
       "  (model): LlamaModel(\n",
       "    (embed_tokens): Embedding(32000, 256)\n",
       "    (layers): ModuleList(\n",
       "      (0-3): 4 x LlamaDecoderLayer(\n",
       "        (self_attn): LlamaSdpaAttention(\n",
       "          (q_proj): Linear(in_features=256, out_features=256, bias=False)\n",
       "          (k_proj): Linear(in_features=256, out_features=128, bias=False)\n",
       "          (v_proj): Linear(in_features=256, out_features=128, bias=False)\n",
       "          (o_proj): Linear(in_features=256, out_features=256, bias=False)\n",
       "          (rotary_emb): LlamaRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): LlamaMLP(\n",
       "          (gate_proj): Linear(in_features=256, out_features=768, bias=False)\n",
       "          (up_proj): Linear(in_features=256, out_features=768, bias=False)\n",
       "          (down_proj): Linear(in_features=768, out_features=256, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): LlamaRMSNorm()\n",
       "        (post_attention_layernorm): LlamaRMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): LlamaRMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=256, out_features=32000, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "model = AutoModelForCausalLM.from_config(config,torch_dtype=torch.float32).to(device)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "77e78c42",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer Name & Parameters\n",
      "----------------------------\n",
      "model.embed_tokens.weight                          | Size: torch.Size([32000, 256])       | Count: 8192000             \n",
      "model.layers.0.self_attn.q_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.0.self_attn.k_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.0.self_attn.v_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.0.self_attn.o_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.0.mlp.gate_proj.weight                | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.0.mlp.up_proj.weight                  | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.0.mlp.down_proj.weight                | Size: torch.Size([256, 768])         | Count: 196608              \n",
      "model.layers.0.input_layernorm.weight              | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.0.post_attention_layernorm.weight     | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.1.self_attn.q_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.1.self_attn.k_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.1.self_attn.v_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.1.self_attn.o_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.1.mlp.gate_proj.weight                | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.1.mlp.up_proj.weight                  | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.1.mlp.down_proj.weight                | Size: torch.Size([256, 768])         | Count: 196608              \n",
      "model.layers.1.input_layernorm.weight              | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.1.post_attention_layernorm.weight     | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.2.self_attn.q_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.2.self_attn.k_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.2.self_attn.v_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.2.self_attn.o_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.2.mlp.gate_proj.weight                | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.2.mlp.up_proj.weight                  | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.2.mlp.down_proj.weight                | Size: torch.Size([256, 768])         | Count: 196608              \n",
      "model.layers.2.input_layernorm.weight              | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.2.post_attention_layernorm.weight     | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.3.self_attn.q_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.3.self_attn.k_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.3.self_attn.v_proj.weight             | Size: torch.Size([128, 256])         | Count: 32768               \n",
      "model.layers.3.self_attn.o_proj.weight             | Size: torch.Size([256, 256])         | Count: 65536               \n",
      "model.layers.3.mlp.gate_proj.weight                | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.3.mlp.up_proj.weight                  | Size: torch.Size([768, 256])         | Count: 196608              \n",
      "model.layers.3.mlp.down_proj.weight                | Size: torch.Size([256, 768])         | Count: 196608              \n",
      "model.layers.3.input_layernorm.weight              | Size: torch.Size([256])              | Count: 256                 \n",
      "model.layers.3.post_attention_layernorm.weight     | Size: torch.Size([256])              | Count: 256                 \n",
      "model.norm.weight                                  | Size: torch.Size([256])              | Count: 256                 \n",
      "lm_head.weight                                     | Size: torch.Size([32000, 256])       | Count: 8192000             \n",
      "----------------------------\n",
      "Total Parameters: 19532032 (19.5 M)\n"
     ]
    }
   ],
   "source": [
    "# 打印模型的每一层及其参数大小\n",
    "def print_model_parameters(model):\n",
    "    print('Layer Name & Parameters')    \n",
    "    print('----------------------------')    \n",
    "    total_params = 0    \n",
    "    for name, parameter in model.named_parameters():    \n",
    "        param_size = parameter.size()        \n",
    "        param_count = torch.prod(torch.tensor(param_size)).item()        \n",
    "        total_params += param_count        \n",
    "        print(f'{name:50} | Size: {str(param_size):30} | Count: {str(param_count):20}')    \n",
    "    print('----------------------------')    \n",
    "    print(f'Total Parameters: {total_params} ({total_params / 1000000:.1f} M)')\n",
    "\n",
    "print_model_parameters(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a6bebacb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Once upon a time,  star hopefully VariableBC shootingética individuals.— Currentlyджа cabinet conflicts sortética Center forg\n"
     ]
    }
   ],
   "source": [
    "def inference(\n",
    "    model: AutoModelForCausalLM,    \n",
    "    tokenizer: AutoTokenizer,    \n",
    "    input_text: str = 'Once upon a time, ',    \n",
    "    max_new_tokens: int = 16\n",
    "):\n",
    "    inputs = tokenizer(input_text, return_tensors='pt').to(device)    \n",
    "    outputs = model.generate(    \n",
    "        **inputs,        \n",
    "        pad_token_id=tokenizer.eos_token_id,        \n",
    "        max_new_tokens=max_new_tokens,        \n",
    "        do_sample=True,        \n",
    "        top_k=40,       \n",
    "        top_p=0.95,        \n",
    "        temperature=0.8    \n",
    "    )    \n",
    "    generated_text = tokenizer.decode( \n",
    "        outputs[0],        \n",
    "        skip_special_tokens=True    \n",
    "    )    \n",
    "    # print(outputs)    \n",
    "    print(generated_text)\n",
    "inference(model, tokenizer)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e59b63b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Kaiming 初始化\n",
    "def kaiming_initialization(model):\n",
    "    for name, param in model.named_parameters():   \n",
    "        if 'weight' in name and param.dim() > 1:            \n",
    "            torch.nn.init.kaiming_uniform_(param, mode='fan_in', nonlinearity='leaky_relu')        \n",
    "        elif 'bias' in name:            \n",
    "            # 一般偏置项可以初始化为 0            \n",
    "            torch.nn.init.constant_(param, 0)\n",
    "        \n",
    "kaiming_initialization(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "47a5e8f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据集\n",
    "from datasets import load_dataset\n",
    "\n",
    "dataset_name_or_path = './TinyStoriesV2'        # 可以替换为本地文件夹路径\n",
    "# ds_train = load_dataset(dataset_name_or_path, split='train')        # 取全部数据\n",
    "ds_train = load_dataset(dataset_name_or_path, split='train[:90%]')    # 只取前 10 %，约 270k 条\n",
    "ds_val = load_dataset(dataset_name_or_path, split='validation[90%:]')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "380ab05e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_func(examples):\n",
    "    max_token = 2048\n",
    "    encoded_texts = tokenizer(examples['text'],add_special_tokens=False)\n",
    "    input_ids_list = encoded_texts['input_ids']\n",
    "    \n",
    "    new_input_ids_list, new_attn_mask_list = [], []\n",
    "    for input_ids in input_ids_list:\n",
    "    \n",
    "        temp = input_ids[-max_token+1:]+[tokenizer.eos_token_id]\n",
    "        new_input_ids_list.append(temp)\n",
    "        new_attn_mask_list.append([1]*len(temp))\n",
    "    return {'input_ids':new_input_ids_list,'attention_mask':new_attn_mask_list}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "e152d3bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ae7f7bc6ab5451389fc7712b7d1f175",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/24866 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "752e639b04b44e85aae6f16cbdd1ed53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2763 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds_train = ds_train.map(\n",
    "process_func,batched=True,num_proc=1,remove_columns=ds_train.column_names,desc='')\n",
    "\n",
    "ds_val = ds_val.map(\n",
    "process_func,batched=True,num_proc=1,remove_columns=ds_val.column_names,desc='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "e7bf281f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "ef8cda3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='saves',                         # 输出路径，包括模型检查点、中间文件等    \n",
    "    overwrite_output_dir=True,                  # 是否覆写 \n",
    "    do_train=True,                              # 是否做训练    \n",
    "    do_eval=True,                               # 是否做评估    \n",
    "    eval_steps=1000,                            # 评估步骤间隔    \n",
    "    per_device_train_batch_size=4,              # 每设备批次    \n",
    "    gradient_accumulation_steps=1,              # 梯度累计步大小，省显存，但小模型没必要，用 1 收敛比较快    \n",
    "    learning_rate=1e-4,                         # 学习率大小    \n",
    "    lr_scheduler_type='cosine',                 # 学习率调度策略，LLM 训练一般都用余弦    \n",
    "    bf16=torch.cuda.is_bf16_supported(),        # 尝试配置 bf16    \n",
    "    fp16=not torch.cuda.is_bf16_supported(),    # bf16 不行就上 fp16    \n",
    "    logging_steps=50,                           # 打印步骤间隔    report_to=None,                             # 日志输出目标，不想用 wandb 可以设置为 None    \n",
    "    num_train_epochs=2,                         # 训练轮数，2 ~ 3 即可    \n",
    "    save_steps=1000,                            # 检查点保存步骤间隔    \n",
    "    save_total_limit=2,                         # output_dir 内留存的检查点最大数目    seed=3407                                   # 随机种子\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "44bb85f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<transformers.trainer.Trainer at 0x28b9bf77850>"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import Trainer\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,                    # 模型实例    \n",
    "    args=training_args,             # 训练参数    \n",
    "    train_dataset=ds_train,         # 训练集    \n",
    "    eval_dataset=ds_val,            # 验证集（评估集）    \n",
    "    tokenizer=tokenizer,            # 分词器    \n",
    "    data_collator=data_collator,    # data collator\n",
    ")\n",
    "trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "b043bb87",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='12434' max='12434' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [12434/12434 10:35, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>9.530900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>7.123300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>6.153700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>5.737500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>5.440000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>5.265400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>5.175500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>4.992700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>4.947900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>4.702600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>4.667800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>4.595800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>4.512100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>4.363300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>4.463400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>4.314800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>4.197300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>4.218000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>4.252300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>4.151900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>4.162900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>4.043600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1150</td>\n",
       "      <td>4.049500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>3.965400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1250</td>\n",
       "      <td>3.957400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>3.950200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1350</td>\n",
       "      <td>3.880100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>3.820300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1450</td>\n",
       "      <td>3.820500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>3.769100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1550</td>\n",
       "      <td>3.750700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>3.679100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1650</td>\n",
       "      <td>3.739500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>3.713200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1750</td>\n",
       "      <td>3.663200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>3.619500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1850</td>\n",
       "      <td>3.609900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>3.611900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1950</td>\n",
       "      <td>3.508100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>3.540800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2050</td>\n",
       "      <td>3.507000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>3.526800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2150</td>\n",
       "      <td>3.499800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>3.520000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2250</td>\n",
       "      <td>3.532700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>3.441800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2350</td>\n",
       "      <td>3.468900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>3.427100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2450</td>\n",
       "      <td>3.395100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>3.460100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2550</td>\n",
       "      <td>3.438300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>3.337600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2650</td>\n",
       "      <td>3.399500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>3.352100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2750</td>\n",
       "      <td>3.364500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>3.312600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2850</td>\n",
       "      <td>3.365600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>3.281100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2950</td>\n",
       "      <td>3.284000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>3.339600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3050</td>\n",
       "      <td>3.218100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3100</td>\n",
       "      <td>3.242500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3150</td>\n",
       "      <td>3.182100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3200</td>\n",
       "      <td>3.228500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3250</td>\n",
       "      <td>3.188000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3300</td>\n",
       "      <td>3.226300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3350</td>\n",
       "      <td>3.161100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3400</td>\n",
       "      <td>3.255900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3450</td>\n",
       "      <td>3.195600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>3.283100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3550</td>\n",
       "      <td>3.249600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3600</td>\n",
       "      <td>3.178000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3650</td>\n",
       "      <td>3.236300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3700</td>\n",
       "      <td>3.214300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3750</td>\n",
       "      <td>3.142800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3800</td>\n",
       "      <td>3.185800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3850</td>\n",
       "      <td>3.143700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3900</td>\n",
       "      <td>3.139100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3950</td>\n",
       "      <td>3.114700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>3.095900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4050</td>\n",
       "      <td>3.158200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4100</td>\n",
       "      <td>3.071300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4150</td>\n",
       "      <td>3.057800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4200</td>\n",
       "      <td>3.042400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4250</td>\n",
       "      <td>3.151400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4300</td>\n",
       "      <td>3.110500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4350</td>\n",
       "      <td>3.038900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4400</td>\n",
       "      <td>3.009300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4450</td>\n",
       "      <td>3.059200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>3.127700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4550</td>\n",
       "      <td>3.053500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4600</td>\n",
       "      <td>3.126800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4650</td>\n",
       "      <td>3.012900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4700</td>\n",
       "      <td>3.007600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4750</td>\n",
       "      <td>3.044600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4800</td>\n",
       "      <td>3.007300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4850</td>\n",
       "      <td>3.054600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4900</td>\n",
       "      <td>2.961200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4950</td>\n",
       "      <td>2.985900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>2.953700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5050</td>\n",
       "      <td>2.969900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5100</td>\n",
       "      <td>3.043800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5150</td>\n",
       "      <td>3.114200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5200</td>\n",
       "      <td>2.918300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5250</td>\n",
       "      <td>3.012300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5300</td>\n",
       "      <td>2.979200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5350</td>\n",
       "      <td>3.016100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5400</td>\n",
       "      <td>2.959500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5450</td>\n",
       "      <td>2.967700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5500</td>\n",
       "      <td>2.941400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5550</td>\n",
       "      <td>2.950700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5600</td>\n",
       "      <td>2.989000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5650</td>\n",
       "      <td>2.930700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5700</td>\n",
       "      <td>2.955000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5750</td>\n",
       "      <td>2.922600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5800</td>\n",
       "      <td>2.961600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5850</td>\n",
       "      <td>3.011800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5900</td>\n",
       "      <td>2.978700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5950</td>\n",
       "      <td>2.966200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>2.963900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6050</td>\n",
       "      <td>2.905800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6100</td>\n",
       "      <td>2.944200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6150</td>\n",
       "      <td>2.942100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6200</td>\n",
       "      <td>3.012300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6250</td>\n",
       "      <td>2.897000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6300</td>\n",
       "      <td>2.943700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6350</td>\n",
       "      <td>2.837700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6400</td>\n",
       "      <td>2.847400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6450</td>\n",
       "      <td>2.889700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6500</td>\n",
       "      <td>2.822800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6550</td>\n",
       "      <td>2.832500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6600</td>\n",
       "      <td>2.868000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6650</td>\n",
       "      <td>2.855600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6700</td>\n",
       "      <td>2.889900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6750</td>\n",
       "      <td>2.853500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6800</td>\n",
       "      <td>2.825800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6850</td>\n",
       "      <td>2.822300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6900</td>\n",
       "      <td>2.868500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6950</td>\n",
       "      <td>2.800500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>2.874100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7050</td>\n",
       "      <td>2.901400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7100</td>\n",
       "      <td>2.774100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7150</td>\n",
       "      <td>2.801700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7200</td>\n",
       "      <td>2.839000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7250</td>\n",
       "      <td>2.794000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7300</td>\n",
       "      <td>2.783200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7350</td>\n",
       "      <td>2.756400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7400</td>\n",
       "      <td>2.868800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7450</td>\n",
       "      <td>2.811400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7500</td>\n",
       "      <td>2.814200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7550</td>\n",
       "      <td>2.841500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7600</td>\n",
       "      <td>2.786800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7650</td>\n",
       "      <td>2.793900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7700</td>\n",
       "      <td>2.826600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7750</td>\n",
       "      <td>2.781000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7800</td>\n",
       "      <td>2.799600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7850</td>\n",
       "      <td>2.750500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7900</td>\n",
       "      <td>2.768300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7950</td>\n",
       "      <td>2.869800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>2.876300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8050</td>\n",
       "      <td>2.701400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8100</td>\n",
       "      <td>2.856400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8150</td>\n",
       "      <td>2.782000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8200</td>\n",
       "      <td>2.742700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8250</td>\n",
       "      <td>2.792000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8300</td>\n",
       "      <td>2.790100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8350</td>\n",
       "      <td>2.750700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8400</td>\n",
       "      <td>2.769400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8450</td>\n",
       "      <td>2.807200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8500</td>\n",
       "      <td>2.819600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8550</td>\n",
       "      <td>2.784000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8600</td>\n",
       "      <td>2.729700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8650</td>\n",
       "      <td>2.786800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8700</td>\n",
       "      <td>2.816300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8750</td>\n",
       "      <td>2.771400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8800</td>\n",
       "      <td>2.714800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8850</td>\n",
       "      <td>2.704000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8900</td>\n",
       "      <td>2.774700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8950</td>\n",
       "      <td>2.780400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>2.795200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9050</td>\n",
       "      <td>2.803800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9100</td>\n",
       "      <td>2.772400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9150</td>\n",
       "      <td>2.774200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9200</td>\n",
       "      <td>2.815800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9250</td>\n",
       "      <td>2.744700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9300</td>\n",
       "      <td>2.839600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9350</td>\n",
       "      <td>2.712300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9400</td>\n",
       "      <td>2.681400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9450</td>\n",
       "      <td>2.744400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9500</td>\n",
       "      <td>2.718400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9550</td>\n",
       "      <td>2.775800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9600</td>\n",
       "      <td>2.759600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9650</td>\n",
       "      <td>2.688500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9700</td>\n",
       "      <td>2.778700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9750</td>\n",
       "      <td>2.684100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9800</td>\n",
       "      <td>2.722400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9850</td>\n",
       "      <td>2.787500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9900</td>\n",
       "      <td>2.726700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9950</td>\n",
       "      <td>2.771200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>2.834300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10050</td>\n",
       "      <td>2.712200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10100</td>\n",
       "      <td>2.775200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10150</td>\n",
       "      <td>2.692400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10200</td>\n",
       "      <td>2.698400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10250</td>\n",
       "      <td>2.743600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10300</td>\n",
       "      <td>2.775800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10350</td>\n",
       "      <td>2.732400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10400</td>\n",
       "      <td>2.745400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10450</td>\n",
       "      <td>2.757100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10500</td>\n",
       "      <td>2.754400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10550</td>\n",
       "      <td>2.747900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10600</td>\n",
       "      <td>2.741300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10650</td>\n",
       "      <td>2.759200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10700</td>\n",
       "      <td>2.805100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10750</td>\n",
       "      <td>2.732100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10800</td>\n",
       "      <td>2.726400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10850</td>\n",
       "      <td>2.772900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10900</td>\n",
       "      <td>2.745400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10950</td>\n",
       "      <td>2.715000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>2.758000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11050</td>\n",
       "      <td>2.737000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11100</td>\n",
       "      <td>2.741200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11150</td>\n",
       "      <td>2.750400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11200</td>\n",
       "      <td>2.743500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11250</td>\n",
       "      <td>2.751600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11300</td>\n",
       "      <td>2.789600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11350</td>\n",
       "      <td>2.621900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11400</td>\n",
       "      <td>2.768200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11450</td>\n",
       "      <td>2.748000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11500</td>\n",
       "      <td>2.738700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11550</td>\n",
       "      <td>2.841300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11600</td>\n",
       "      <td>2.718300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11650</td>\n",
       "      <td>2.814500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11700</td>\n",
       "      <td>2.728000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11750</td>\n",
       "      <td>2.720100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11800</td>\n",
       "      <td>2.842800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11850</td>\n",
       "      <td>2.740900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11900</td>\n",
       "      <td>2.719600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11950</td>\n",
       "      <td>2.756500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12000</td>\n",
       "      <td>2.755100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12050</td>\n",
       "      <td>2.685700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12100</td>\n",
       "      <td>2.728600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12150</td>\n",
       "      <td>2.753700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12200</td>\n",
       "      <td>2.743000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12250</td>\n",
       "      <td>2.767600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12300</td>\n",
       "      <td>2.675600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12350</td>\n",
       "      <td>2.703700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12400</td>\n",
       "      <td>2.777900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=12434, training_loss=3.1741219407514647, metrics={'train_runtime': 635.7082, 'train_samples_per_second': 78.231, 'train_steps_per_second': 19.559, 'total_flos': 1023343130692608.0, 'train_loss': 3.1741219407514647, 'epoch': 2.0})"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "trainer.train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "725e5257",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "today,  wife had how for it an Jane dreamb\". was a big, even. There was very small and loved the knot's favorite voice. They were having so much fun.\n",
      "One day, the day, Momo was playing with a big, shiny thing to play with the shiny car. She saw a pretty picture of a picture of the stick. The man was sad and said, \"Bird, let me take it!\"\n",
      "So, the dog and the cat was\n"
     ]
    }
   ],
   "source": [
    "def inference(\n",
    "    model: AutoModelForCausalLM,    \n",
    "    tokenizer: AutoTokenizer,    \n",
    "    input_text: str = 'today, ',    \n",
    "    max_new_tokens: int = 16\n",
    "):\n",
    "    inputs = tokenizer(input_text, return_tensors='pt').to(device)    \n",
    "    outputs = model.generate(    \n",
    "        **inputs,        \n",
    "        pad_token_id=tokenizer.eos_token_id,        \n",
    "        max_new_tokens=max_new_tokens,        \n",
    "        do_sample=True,        \n",
    "        top_k=40,       \n",
    "        top_p=0.95,        \n",
    "        temperature=0.8    \n",
    "    )    \n",
    "    generated_text = tokenizer.decode( \n",
    "        outputs[0],        \n",
    "        skip_special_tokens=True    \n",
    "    )    \n",
    "    # print(outputs)    \n",
    "    print(generated_text)\n",
    "inference(model, tokenizer, max_new_tokens=100)\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
